from urllib.request import urlopen
import re
import pandas as pd
import csv
import matplotlib.pyplot as plt

userName = ""  # 姓名
deptName = ""  # 所属学部
title = ""  # 学术称号
birthday = ""  # 出生年月
birthplace = ""  # 出生地
date = ""  # 当选年份
area = ""  # 专业领域
img = ""  # 图片url
Message = []  # 记录数据
num = 0  # 记录总人数


def page_err():
    error = "暂无信息"
    Message.append({"姓名": userName,
                    "所属部门": deptName,
                    "学术称号": error,
                    "专业领域": error,
                    "出生年月": error,
                    "出生地": error,
                    "当选年份": error,
                    "照片url": error})
    return


def remove_word(text):
    f_remove = open("去除词.txt", mode="r", encoding="utf-8")
    remove = f_remove.read()
    f_remove.close()

    remove = remove.replace('\n', '?')
    return re.sub(remove, '', text)


# 打开一个网页并使用re提取页面信息，返回一个迭代器
def webpage_open(re_obj, url_page):
    resp = urlopen(url_page)  # 打开网页，并将响应结果返回resp

    # 提取每个部门的信息
    department_it = re_obj.finditer(resp.read().decode("utf-8"))

    # 释放resp
    del resp

    # 返回提取到的信息
    return department_it


#
#
#
# 第一部分: 打开每一个院士的介绍页面
# re正则表达式预加载
dept_pattern = '''<div class="xunhuan">.*?<b>(?P<deptName>.*?)</b>.*?<div class="rmbs_a">(?P<users>.*?)</div>'''
users_pattern = '''<a href="(?P<userMsg_url>.*?)".*?target="_blank">(?P<userName>.*?)</a>'''
userMsg_pattern01 = r'''<p align="justify".*?</p>'''
userMsg_pattern02 = r'''TRS_Editor LI.*?TRS_Editor A.*?</style>.*?</p>'''

reObj_department = re.compile(dept_pattern, re.S)
reObj_users = re.compile(users_pattern, re.S)
reObj_userMsg01 = re.compile(userMsg_pattern01, re.S)
reObj_userMsg02 = re.compile(userMsg_pattern02, re.S)

# 全体院士名单页面
url = "https://casad.cas.cn/ysxx2022/ysmd/qtys/"

# 将每个部门进行遍历
for dept in webpage_open(reObj_department, url):
    # 提取部门名称
    deptName = dept.group("deptName")

    # 提取每个部门内每个成员,此时提取的信息只包含成员信息的url和成员名称
    user_it = reObj_users.finditer(dept.group("users"))

    # 遍历该部中每个成员
    for user in user_it:
        # 提取成员名
        userName = user.group("userName")

        # 提取每个院士信息页面的url链接
        user_url = user.group("userMsg_url")

        # 进入成员信息页面
        resp_user = urlopen(user_url)

        # 提取成员介绍信息
        userMsg_text = ""  # 用于存放经过去标签处理的文本
        userText_str = str(resp_user.read().decode("utf-8"))  # 原文本

        # 对提取到的原页面信息进行初步处理
        for reObj_userMsg in [reObj_userMsg01, reObj_userMsg02]:
            userMsg_search = reObj_userMsg.search(userText_str)
            # 去标签化处理
            try:
                label_remove = "<.*?>"
                userMsg_text = re.sub(label_remove, '', userMsg_search.group())
                break
            except Exception:
                continue
        else:
            page_err()
            continue

        # 从原页面文本中提取图片信息
        reObj_img = re.compile('''<div class="acadImg">.*?(?P<img>W.*?.(jpg|png))"''')  # 提取图片网址信息
        reObj_pf = re.compile('''^.*/''')  # 提取图片url前缀
        try:
            img_prefix = reObj_pf.search(user_url).group()
            img_main = reObj_img.search(userText_str).group("img")
            img = img_prefix + img_main
        except AttributeError:
            img = "暂无图片信息"

        #
        #
        #
        # 第二部分: 对提取到的文本进行分析
        # 1.提取学术称号
        re_title01 = '''(?P<title>[\u4e00-\u9fa5]+家)[，。,]'''
        re_title02 = '''(?P<title>[\u4e00-\u9fa5]+学)[，。,]'''

        # 匹配两个正则表达式
        for re_title in [re_title01, re_title02]:
            reObj_title = re.compile(re_title)
            title_re = reObj_title.search(userMsg_text)
            try:
                title = title_re.group("title")
                break
            except Exception:
                continue
        else:
            title = "暂无学术称号"

        # 2.提取专业领域
        re_short_text = '''(学|家)[。，,](?P<short>.*?)[。，].*?(出)?生(于)?'''
        reObj_short_text = re.compile(re_short_text)
        try:
            short_text = reObj_short_text.search(userMsg_text).group("short")
        except AttributeError:
            short_text = "暂无专业领域信息"

        area = remove_word(short_text)

        # 3.提取出生年月及出生地

        birth_re1 = '''[。](?P<birthday>[\d年月]+).*?生于(?P<birthplace>.*?)[，。,.]'''
        birth_re2 = '''(?P<birthday>[\d年月]+)出?生[，。,.](?P<birthplace>.*?)人[。，.,]'''
        birth_re3 = '''[，](?P<birthday>[\d年月]+).*?生于(?P<birthplace>.*?)[，。,.]'''
        birth_re4 = '''[，。,.](?P<birthday>[\d年月日]+)出生'''

        for birth_re in [birth_re1, birth_re2, birth_re3]:
            reObj_birth = re.compile(birth_re)

            birth = reObj_birth.search(userMsg_text)
            if birth is None:
                continue
            else:
                birthday = birth.group("birthday")
                birthplace = birth.group("birthplace")
                break
        else:
            reObj_no_place = re.compile(birth_re4)
            birthday = reObj_no_place.search(userMsg_text).group("birthday")
            birthplace = "暂无信息"

        # 4.提取当选年份
        date_re = '''[，。,.](?P<date>\d+年?)当选'''

        reObj_date = re.compile(date_re)
        try:
            date = reObj_date.search(userMsg_text).group("date")
        except AttributeError:
            date = "提取当选年份失败!\n\n"

        # 记录数据
        Message.append({"姓名": userName,
                        "所属部门": deptName,
                        "学术称号": title,
                        "专业领域": area,
                        "出生年月": birthday,
                        "出生地": birthplace,
                        "当选年份": date,
                        "照片url": img})
        num = num + 1

        # 进度
        if num % 5 == 0 or num == 853:
            print("已完成：" + "{:.2f}".format(num * 100 / 853) + "%")
print("院士信息提取完成！")

# 创建csv表格记录数据
with open("院士统计表.csv", mode="w", encoding="utf-8", newline='') as f:
    headers = ["姓名", "所属部门", "学术称号", "专业领域", "出生年月", "出生地", "当选年份", "照片url"]  # 定义表头
    writer = csv.DictWriter(f, headers)

    writer.writeheader()  # 将表头写入
    writer.writerows(Message)

# 统计出生地省份数
# 使用re正则表达式提取出生地省份
provinces = ""
f_provinces = open("省份.txt", mode="r", encoding="utf-8")
provinces = f_provinces.read()
provinces = provinces.replace('\n', '?')
reObj_province = re.compile(provinces)

# 将省会城市改为省
cities = {
    "石家庄": "河北",
    "太原": "山西",
    "西安": "陕西",
    "济南": "山东",
    "郑州": "河南",
    "丹东": "辽宁",
    "长春": "吉林",
    "哈尔滨": "黑龙江",
    "南京": "江苏",
    "杭州": "浙江",
    "合肥": "安徽",
    "南昌": "江西",
    "福州": "福建",
    "武汉": "湖北",
    "长沙": "湖南",
    "成都": "四川",
    "贵阳": "贵州",
    "昆明": "云南",
    "广州": "广东",
    "海口": "海南",
    "兰州": "甘肃",
    "西宁": "青海",
    "内蒙": "内蒙古"
}

# 提取地点到places_list集合中
places_list = []
place_province = ""
for msg in Message:
    place = msg.get("出生地")  # 从Message中提取出生地信息

    try:
        place_province = reObj_province.search(place).group()  # 提取出省份
    except Exception:

        place_province = place[:2]

    # place_city==place时说明出生地名称可能是省会,将其转化为省份
    if place_province == place:

        for city in cities.keys():
            if city in place_province:
                place_province = cities.get(city)
                break
    if place_province == '':
        continue
    places_list.append(place_province)

counts = pd.Series(places_list).value_counts()

# 绘制直方图
plt.figure(figsize=(12, 6))
plt.rcParams['font.family'] = 'Times New Roman, SimSun'
plt.bar(list(counts.keys()), list(counts))
plt.xlabel("省份", fontsize=14)
plt.ylabel("院士人数", fontsize=14)
plt.title("院士出生地分布直方图", fontsize=16)
plt.xticks(rotation=45, fontsize=12)
plt.savefig("院士出生地分布.png", dpi=300)
plt.show()
print("已生成'院士出生地分布直方图'")
